import pandas as pd
import numpy as np
# 文件路径
datafile_test = './data/test.csv'  # test原始数据，第一行为属性标签
datafile_train = './data/train.csv'  # train原始数据，第一行为属性标签

# 读取训练样本和测试样本
data_train = pd.read_csv(datafile_train)
data_test = pd.read_csv(datafile_test)

# 数据清洗
# 数据清洗后数据表的存放位置
cleanfile_train = './tmp/clean_train_徐伟杰.csv'
cleanfile_test = './tmp/clean_test_徐伟杰.csv'

# 训练样本和测试样本进行合并，方便数据清洗
data = pd.concat([data_train, data_test], axis=0, join='outer')

# 对整个表的空值进行处理
# 原数据中缺失值为 null 字符串，设置为 numpy.nan
data.iloc[:, :5] = data.iloc[:, :5].applymap(
    lambda x:np.nan if x == 'null' else x
)

# 对日期类型的空值设置为None，方便后面整列转成时间类型
data.iloc[:, 5:] = data.iloc[:, 5:].applymap(
    lambda x:None if x == 'null' else x
)
# 处理date_received字段
data['date_received'] = data['date_received'].astype('str').apply(
    lambda x:x.split('.')[0]
)
# 转成datetime格式
data['date_received'] = pd.to_datetime(data['date_received'])

# 处理date字段
data['date'] = data['date'].astype('str').apply(
    lambda x:x.split('.')[0]
)
data['date'] = pd.to_datetime(data['date'])

# 满减优惠改写成折扣形式
data['discount_rate'] = data['discount_rate'].fillna('null')
def discount(x):
    if ':' in x:
        split_item = x.split(':')
        # 计算折扣率：举个例子：30:5 => ((30-5)/30) = 0.83
        discount_rate = (int(split_item[0]) - int(split_item[1]))/int(split_item[0])
        return round(discount_rate, 2)
    elif x == 'null':
        return np.nan
    else:
        return float(x)

data['discount_rate'] = data['discount_rate'].map(discount)

# 根据领券月份，提取清洗后的训练样本和测试样本
received_month = data['date_received'].apply(lambda x: x.month)
received_month.value_counts()
clean_train = data.loc[received_month != 7, :]  # 提取清洗后训练样本
clean_test = data.loc[received_month == 7, :]  # 提取清洗后测试样本
# clean_test.drop('date', axis=1, inplace=True)  # 删除date列

# 导出数据
# clean_train.to_csv(cleanfile_train, index=False)
# clean_test.to_csv(cleanfile_test, index=False)

# 用户、商户、优惠券的特征结果表
userfile = './tmp/data_user.csv'
merchantfile = './tmp/data_merchant.csv'
couponfile = './tmp/data_coupon.csv'
train_quality = clean_train.copy()
test_quality = clean_test.copy()

# 导入自定义用户、商户、优惠券的特征包
from feature_name import feature_name
data_user, data_merchant, data_coupon = feature_name(train_quality=train_quality)
data_user.to_csv(userfile, index=False)  # 导出data_user表
data_merchant.to_csv(merchantfile, index=False)  # 导出data_merchant数据表
data_coupon.to_csv(couponfile, index=False)   #导出data_coupon数据表

# 对训练样本与指标类型表进行拼接
train_merge = pd.merge(data_user, train_quality, on='user_id')
train_merge = pd.merge(train_merge, data_merchant, on="merchant_id")
train_merge = pd.merge(train_merge, data_coupon, on='coupon_id', how='left')
train_merge.isnull().sum()  # 统计缺失值
train_merge.iloc[:,-2:] = train_merge.iloc[:, -2:].fillna(0)  # 缺失值填充
# print('构建指标后训练样本的形状：', train_merge.shape())
trainfile = './tmp/train_cleaned.csv'  # 导出数据
train_merge.to_csv(trainfile, index=False)

# 对测试样本与指标类型表进行拼接
test_merge = pd.merge(test_quality, data_user, on='user_id')
test_merge = pd.merge(test_merge, data_merchant, on='merchant_id')
test_merge = pd.merge(test_merge, data_coupon, on='coupon_id', how='left')
test_merge.isnull().sum()  # 统计缺失值
test_merge.iloc[:,-2:] = test_merge.iloc[:,-2:].fillna(0)  # 缺失值填充
# print('构建指标后测试样本的形状：', test_merge.shape())
testfile = './tmp/test_cleaned.csv'  # 导出数据
test_merge.to_csv(testfile, index=False)



# 代码7-10

# 建立训练样本分类标签
train_merge["class"] = 0  # 标签0
train_merge.loc[(train_merge['date']-
                 train_merge['date_received']).dt.days<=15, 'class']=1  # 标签1

# 删除非正负样本的数据（用户未领券的记录）
print(train_merge.shape)
train_merge = train_merge[train_merge['coupon_id'].notnull()]
print(train_merge.shape)
trainfile_class = './tmp/train_class.csv'  # 导出数据
train_merge.to_csv(trainfile_class, index=False)
